# -*- coding: utf-8 -*-
# ----------------------------
# @Time    : 2021/6/26 上午11:15
# @Author  : changqingai
# @FileName: pyspark_demo.py
# ----------------------------

from pyspark import SparkContext

if __name__ == "__main__":
    sc = SparkContext(master="local[*]", appName="pyspark-demo")
    lines = sc.textFile("../datasets/wc/1.txt")
    res = lines.flatMap(lambda line: line.split(" "))\
        .map(lambda word: (word, 1))\
        .reduceByKey(lambda a, b: a + b)\
        .collect()
    print('res', res)
    print(type(res))
    for (word, count) in res:
        print(word, count)
